/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package cz.mzk.oai.metadataRepository.analysis.marcxml;

import cz.mzk.oai.metadataRepository.tools.CannotReadDirectoryException;
import cz.mzk.oai.metadataRepository.repository.RecordAnalyzer;
import cz.mzk.oai.metadataRepository.metadata.MarcRecord;
import cz.mzk.oai.metadataRepository.metadata.MultipleDataField;
import cz.mzk.oai.metadataRepository.tools.FileTools;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileWriter;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.logging.FileHandler;
import java.util.logging.Handler;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.logging.SimpleFormatter;
import org.dom4j.DocumentException;

/**
 *
 * @author Martin Rehanek, Martin.Rehanek@gmail.com
 */
public class MarcxmlFilesAnalyzer {

    public static void analyzeDirectory(String dirName) {
        File mainDir = new File(dirName);
        if (!mainDir.isDirectory()) {
            System.err.println(dirName + " is not directory");
            return;
        }
        int mollMapCounter = 0;
        int mollRkpCounter = 0;
        int oldMapCounter = 0;
        int counter = 0;
        RecordAnalyzer ra = new RecordAnalyzer();
        List<String> fileNames = null;
        try {
            fileNames = FileTools.readFilenames(mainDir);
        } catch (CannotReadDirectoryException ex) {
            Logger.getLogger(MarcxmlFilesAnalyzer.class.getName()).log(Level.SEVERE, null, ex);
        }
        for (String fileName : fileNames) {
            counter++;
            String absoluteFileName = mainDir.getAbsolutePath() + File.separator + fileName;
            File file = new File(absoluteFileName);
            if (file.isFile()) {
                MarcRecord record = null;
                try {
                    record = new MarcRecord(file);
                    mollMapCounter += ra.isMollMap(record) ? 1 : 0;
                    mollRkpCounter += ra.isMollRkp(record) ? 1 : 0;
                    oldMapCounter += ra.isOldMap(record) ? 1 : 0;
                } catch (DocumentException ex) {
                    System.err.println("Error parsing file " + fileName);
                } catch (FileNotFoundException e) {
                    System.err.println("file " + fileName + " not found");
                }
            }
            if (counter % 1000 == 0) {
                System.out.println("Processed " + counter + " files");
                System.out.println("MollMap: " + mollMapCounter);
                System.out.println("MollRkp: " + mollRkpCounter);
                System.out.println("OldMaps: " + oldMapCounter);
            }
        }
        System.out.println("Processed " + counter + " files");
        System.out.println("MollMap: " + mollMapCounter);
        System.out.println("MollRkp: " + mollRkpCounter);
        System.out.println("OldMaps: " + oldMapCounter);
    }

    public static void schramTest(String dir) {
        RecordAnalyzer analyzer = new RecordAnalyzer();
        File mainDir = new File(dir);
        if (!mainDir.isDirectory()) {
            System.err.println(dir + " is not directory");
            return;
        }
        List<String> fileNames = null;
        try {
            fileNames = FileTools.readFilenames(mainDir);
        } catch (CannotReadDirectoryException ex) {
            Logger.getLogger(MarcxmlFilesAnalyzer.class.getName()).log(Level.SEVERE, null, ex);
        }
        //System.out.println("files: " + fileNames.size());
        int counter = 0;
        boolean schramIn856 = false;
        boolean schramSkr = false;

        int schramIn856Counter = 0;
        int schramSkrCounter = 0;
        int schramGrafListyAnd650Ved = 0;
        int schramGrafListyAnd650Por = 0;
        int schramGrafListyAnd650N = 0;

        Logger logger = Logger.getLogger("schramGrafListyAnd650NLoger");
        Handler fileHandler = null;
        try {
            fileHandler = new FileHandler("/home/martin/tmp/schramGrafListyAnd650NLoger.log", true);
        } catch (IOException ex) {
            Logger.getLogger(MarcxmlFilesAnalyzer.class.getName()).log(Level.SEVERE, null, ex);
        } catch (SecurityException ex) {
            Logger.getLogger(MarcxmlFilesAnalyzer.class.getName()).log(Level.SEVERE, null, ex);
        }
        fileHandler.setFormatter(new SimpleFormatter());
        logger.addHandler(fileHandler);

        Set<String> valuesIn655a = new HashSet<String>();
        Set<String> valuesIn655ATogether = new HashSet<String>();
        Map<String, Integer> valuesIn655ACount = new HashMap<String, Integer>(5);
        Map<String, Integer> valuesIn655ACountTogether = new HashMap<String, Integer>(5);
        Set<String> idsOfEmpty655 = new HashSet<String>(29);
        Set<String> idsOfSchramGrafListyAnd650N = new HashSet<String>(202);
        Set<String> idsOfHistMaps = new HashSet<String>(4);
        Set<String> idsOfHistMapAndGrafList = new HashSet<String>(1);
        Set<String> idsOfFoto = new HashSet<String>(1);
        Set<String> idsOfFotoFoto = new HashSet<String>(1);



        for (String fileName : fileNames) {
            String absoluteFileName = mainDir.getAbsolutePath() + File.separator + fileName;
            //System.out.println("file: " + absoluteFileName);
            File file = new File(absoluteFileName);
            if (file.isFile()) {
                MarcRecord record = null;
                try {
                    record = new MarcRecord(file);
                } catch (FileNotFoundException e) {
                    System.err.println("file " + fileName + " not found");
                } catch (DocumentException ex) {
                    System.err.println("Error parsing file " + fileName);
                }
                counter++;
                if (counter % 1000 == 0) {
                    System.out.println("Processed " + counter + " files");
                    System.out.println("SchramIn856: " + schramIn856Counter);
                    System.out.println("SchramAccordingTo910: " + schramSkrCounter);
                    System.out.println("Values in 655a:");
                    /*for (String value : valuesIn655a) {
                    System.out.print("S:");
                    System.out.println(value + ":" + valuesIn655ACount.get(value));
                    }*/
                    for (String value : valuesIn655ATogether) {
                        System.out.print("T:");
                        System.out.println(value + ":" + valuesIn655ACountTogether.get(value));
                    }
                    System.out.println("schramGrafListyAnd650Ved: " + schramGrafListyAnd650Ved);
                    System.out.println("schramGrafListyAnd650Por: " + schramGrafListyAnd650Por);
                    System.out.println("schramGrafListyAnd650N: " + schramGrafListyAnd650N);
                    System.out.println("----------------------------------");

                }
                //System.out.println("856:" + record.getDatafield856().getSubfield('x'));
                schramIn856 = record.getDatafield856().subfieldsContain('x', "schram");
                schramSkr = analyzer.isSchram(record);
                if (analyzer.isSchram(record)) {
                    MultipleDataField datafield655 = record.getDatafield655();
                    String valueATogether = datafield655.subfieldsToString('a');
                    valuesIn655ATogether.add(valueATogether);
                    Integer valueCounterTogether = valuesIn655ACountTogether.get(valueATogether);
                    valuesIn655ACountTogether.put(valueATogether, valuesIn655ACountTogether.containsKey(valueATogether) ? ++valueCounterTogether : Integer.valueOf(1));
                    for (String valueA : datafield655.getSubfieldContents('a')) {
                        valuesIn655a.add(valueA);
                        Integer valueCounter = valuesIn655ACount.get(valueA);
                        valuesIn655ACount.put(valueA, valuesIn655ACount.containsKey(valueA) ? ++valueCounter : Integer.valueOf(1));
                    }

                    switch (datafield655.size()) {
                        case 0:
                            idsOfEmpty655.add(record.getId());
                            break;
                        case 1:
                            if (datafield655.subfieldsContain('a', "historické mapy")) {
                                idsOfHistMaps.add(record.getId());
                            } else if (datafield655.subfieldsContain('a', "fotografie")) {
                                idsOfFoto.add(record.getId());
                            } else if (datafield655.subfieldsContain('a', "grafické listy")) {
                                MultipleDataField datafield650 = record.getDatafield650();
                                if (datafield650.subfieldsContain('a', "Schramova sbírka vedut")) {
                                    schramGrafListyAnd650Ved++;
                                } else if (datafield650.subfieldsContain('a', "Schramova sbírka portrétů")) {
                                    schramGrafListyAnd650Por++;
                                } else {
                                    //logger.log(Level.WARNING, record.getId());
                                    schramGrafListyAnd650N++;
                                    idsOfSchramGrafListyAnd650N.add(record.getId());
                                }
                            }
                            break;
                        case 2:
                            if (datafield655.subfieldsContain('a', "historické mapy") && datafield655.subfieldsContain('a', "grafické listy")) {
                                idsOfHistMapAndGrafList.add(record.getId());
                            }
                            List<String> subfields = datafield655.getSubfieldContents('a');
                            if (subfields.get(0).equals("fotografie") && subfields.get(1).equals("fotografie")) {
                                idsOfFotoFoto.add(record.getId());
                            }
                            break;

                    }
                }
                schramIn856Counter += schramIn856 ? 1 : 0;
                schramSkrCounter += schramSkr ? 1 : 0;

            }
        }
        fileHandler.close();
        System.out.println("Files: " + counter);
        System.out.println("SchramIn856: " + schramIn856Counter);
        System.out.println("SchramAccordingTo910: " + schramSkrCounter);
        System.out.println("Values in 655a:");
        for (String value : valuesIn655a) {
            System.out.print("S:");
            System.out.println(value + ":" + valuesIn655ACount.get(value));
        }
        for (String value : valuesIn655ATogether) {
            System.out.print("T:");
            System.out.println(value + ":" + valuesIn655ACountTogether.get(value));
        }
        System.out.println("schramGrafListyAnd650Ved: " + schramGrafListyAnd650Ved);
        System.out.println("schramGrafListyAnd650Por: " + schramGrafListyAnd650Por);
        System.out.println("schramGrafListyAnd650N: " + schramGrafListyAnd650N);

        //Seznam id těch záznamů, které mají v 655a 'grafické listy', ale nemají nic v 650
        StringBuilder builder = new StringBuilder();
        builder.append("Seznam id těch záznamů, které mají v 655a 'grafické listy', ale nemají nic v 650:\n");
        for (String id : idsOfSchramGrafListyAnd650N) {
            builder.append(id);
            builder.append('\n');
        }
        logger.log(Level.SEVERE, builder.toString());

        //Seznam id těch záznamů, které nemají nic v 655a\n
        builder = new StringBuilder();
        builder.append("Seznam id těch záznamů, které nemají nic v 655a\n");
        for (String id : idsOfEmpty655) {
            builder.append(id);
            builder.append('\n');
        }
        logger.log(Level.SEVERE, builder.toString());

        //Seznam id těch záznamů, maji v 655a 'historicke mapy'
        builder = new StringBuilder();
        builder.append("Seznam id těch záznamů, maji v 655a 'historicke mapy'\n");
        for (String id : idsOfHistMaps) {
            builder.append(id);
            builder.append('\n');
        }
        logger.log(Level.SEVERE, builder.toString());

        //Seznam id těch záznamů, maji v 655a 'historicke mapy' a 'grafické listy'
        builder = new StringBuilder();
        builder.append("Seznam id těch záznamů, maji v 655a 'historicke mapy' a 'grafické listy'\n");
        for (String id : idsOfHistMapAndGrafList) {
            builder.append(id);
            builder.append('\n');
        }
        logger.log(Level.SEVERE, builder.toString());

        //Seznam id těch záznamů, maji v 655a 'fotografie'
        builder = new StringBuilder();
        builder.append("Seznam id těch záznamů, maji v 655a 'fotografie'\n");
        for (String id : idsOfFoto) {
            builder.append(id);
            builder.append('\n');
        }
        logger.log(Level.SEVERE, builder.toString());

        //Seznam id těch záznamů, maji v 655a 'fotografie' a 'fotografie'
        builder = new StringBuilder();
        builder.append("Seznam id těch záznamů, maji v 655a 'fotografie' a 'fotografie'\n");
        for (String id : idsOfFotoFoto) {
            builder.append(id);
            builder.append('\n');
        }
        logger.log(Level.SEVERE, builder.toString());

    }

    /*
     * Vypise, kolik metadatovych souboru ma neco v datafieldu 651
     */
    public static void datafield651Analysis(String dir, String logFile) {
        File mainDir = new File(dir);
        if (!mainDir.isDirectory()) {
            System.err.println(dir + " is not directory");
            return;
        }
        try {
            OutputStreamWriter writer = new FileWriter(logFile, true);
            List<String> fileNames = null;
            try {
                fileNames = FileTools.readFilenames(mainDir);
            } catch (CannotReadDirectoryException ex) {
                Logger.getLogger(MarcxmlFilesAnalyzer.class.getName()).log(Level.SEVERE, null, ex);
            }
            int counter = 0;
            for (String fileName : fileNames) {
                File recordFile = new File(mainDir.getAbsoluteFile() + File.separator + fileName);
                try {
                    MarcRecord record = new MarcRecord(recordFile);
                    if (!record.getDatafield651().empty()) {
                        counter++;
                    } else {
                        writer.append(record.getId() + '\n');
                    }
                } catch (DocumentException ex) {
                    System.err.println("Error while reading file " + recordFile.getAbsolutePath());
                } catch (FileNotFoundException ex) {
                    System.err.println("File " + recordFile.getAbsolutePath() + " not found");
                }
            }
            writer.close();
            System.out.println("Directory: " + mainDir);
            System.out.println("Number of files: " + fileNames.size());
            System.out.println("Files that have 651 datafield: " + counter);
        } catch (IOException ex) {
            Logger.getLogger(MarcxmlFilesAnalyzer.class.getName()).log(Level.SEVERE, null, ex);
        }
    }

    /*
     * Vypise, kolik metadatovych souboru ma neco v datafieldu 651
     */
    public static void analyzeDatafield856(String dir, String logFile) {
        File mainDir = new File(dir);
        if (!mainDir.isDirectory()) {
            System.err.println(dir + " is not directory");
            return;
        }
        try {
            OutputStreamWriter writer = new FileWriter(logFile, true);
            List<String> fileNames = null;
            try {
                fileNames = FileTools.readFilenames(mainDir);
            } catch (CannotReadDirectoryException ex) {
                Logger.getLogger(MarcxmlFilesAnalyzer.class.getName()).log(Level.SEVERE, null, ex);
            }
            int success = 0;
            int counter = 0;
            //System.out.println("files: " + fileNames.size());
            for (String fileName : fileNames) {
                if (++counter % 100 == 0) {
                    //System.out.println("processed " + counter + " files");
                }
                File recordFile = new File(mainDir.getAbsoluteFile() + File.separator + fileName);
                try {
                    MarcRecord record = new MarcRecord(recordFile);
                    if (record.getDatafield856().size() != 0) {
                        success++;
                    } else {
                        writer.append(record.getId() + '\n');
                    }
                } catch (DocumentException ex) {
                    System.err.println("Error while reading file " + recordFile.getAbsolutePath());
                } catch (FileNotFoundException ex) {
                    System.err.println("File " + recordFile.getAbsolutePath() + " not found");
                }
            }
            writer.close();
            System.out.println("Directory: " + mainDir);
            System.out.println("Number of files: " + fileNames.size());
            System.out.println("Files that have datafield 856: " + success);
        } catch (IOException ex) {
            Logger.getLogger(MarcxmlFilesAnalyzer.class.getName()).log(Level.SEVERE, null, ex);
        }
    }

    public static void analyzeDatafield040b(String dir, String logFile) {
        File mainDir = new File(dir);
        if (!mainDir.isDirectory()) {
            System.err.println("'" + dir + "' is not directory");
            return;
        }
        List<String> fileNames = null;
        try {
            fileNames = FileTools.readFilenames(mainDir);
        } catch (CannotReadDirectoryException ex) {
            Logger.getLogger(MarcxmlFilesAnalyzer.class.getName()).log(Level.SEVERE, null, ex);
        }

        int counter = 0;
        MarcDataFieldStatistics dataFieldAnalyzer = new MarcDataFieldStatistics("040", 'b');
        for (String fileName : fileNames) {
            if (++counter % 100 == 0) {
                //System.out.println("processed " + counter + " files");
            }
            File recordFile = new File(mainDir.getAbsoluteFile() + File.separator + fileName);
            try {
                AnalyticMarcRecord record = new AnalyticMarcRecord(recordFile);
                MultipleDataField datafield = record.getDatafield040();
                dataFieldAnalyzer.addNewDatafield(datafield, record.getId());
                //analyzeDatafieldOccurence(datafield, "040", 'b');
            } catch (DocumentException ex) {
                System.err.println("Error while reading file " + recordFile.getAbsolutePath());
            } catch (FileNotFoundException ex) {
                System.err.println("File " + recordFile.getAbsolutePath() + " not found");
            }
        }
        System.out.println("Directory: " + mainDir);
        System.out.println("Number of files: " + fileNames.size());
        dataFieldAnalyzer.printResults();
    }

    /**
     * @param dir
     * @param logFile
     * v poli 072 Rukopisy, v poli 655 rukopisy, v poli 655 (není :) mikrodokumenty,
     * v poli 533 (není :) mikrofilm
     */
    public static void manuscriptTest(String dir) {
        File mainDir = new File(dir);
        if (!mainDir.isDirectory()) {
            System.err.println(dir + " is not directory");
            return;
        }
        List<String> fileNames = null;
        try {
            fileNames = FileTools.readFilenames(mainDir);
        } catch (CannotReadDirectoryException ex) {
            Logger.getLogger(MarcxmlFilesAnalyzer.class.getName()).log(Level.SEVERE, null, ex);
        }
        int success = 0;
        int counter = 0;
        boolean d072sXvRukopisy = false;
        boolean d655sAvRukopisy = false;
        boolean d655sAvMikrodokumenty = false;
        boolean d533sAvMikrofilm = false;

        int cnt072RukA655Ruk = 0;
        int cnt072RukA655NRuk = 0;
        int cnt072NRukA655Ruk = 0;

        int cnt072RukA655RukA655NMik = 0;
        int cnt072RukA655RukA533NMik = 0;

        for (String fileName : fileNames) {
            if (++counter % 10000 == 0) {
                System.out.println("processed " + counter + " files");
            }
            File recordFile = new File(mainDir.getAbsoluteFile() + File.separator + fileName);
            try {
                MarcRecord record = new MarcRecord(recordFile);
                MultipleDataField datafield072 = record.getDatafield072();
                MultipleDataField datafield655 = record.getDatafield655();
                MultipleDataField datafield533 = record.getDatafield533();
                d072sXvRukopisy = datafield072.contains('x', "Rukopisy");
                d655sAvRukopisy = datafield655.contains('a', "Rukopisy");
                d655sAvMikrodokumenty = datafield655.contains('a', "Mikrodokumenty");
                d533sAvMikrofilm = datafield533.contains('a', "Mikrofilm");
                if (d072sXvRukopisy && d655sAvRukopisy && !d655sAvMikrodokumenty && !d533sAvMikrofilm) {
                    success++;
                }
                if (d072sXvRukopisy && d655sAvRukopisy) {
                    cnt072RukA655Ruk++;
                } else if (d072sXvRukopisy && !d655sAvRukopisy) {
                    cnt072RukA655NRuk++;
                } else if (!d072sXvRukopisy && d655sAvRukopisy) {
                    cnt072NRukA655Ruk++;
                }

                if (d072sXvRukopisy && d655sAvRukopisy && !d655sAvMikrodokumenty) {
                    cnt072RukA655RukA655NMik++;
                }
                if (d072sXvRukopisy && d655sAvRukopisy && !d533sAvMikrofilm) {
                    cnt072RukA655RukA533NMik++;
                }

            } catch (DocumentException ex) {
                System.err.println("Error while reading file " + recordFile.getAbsolutePath());
            } catch (FileNotFoundException ex) {
                System.err.println("File " + recordFile.getAbsolutePath() + " not found");
            }
        }
        System.out.println("Directory: " + mainDir);
        System.out.println("Number of files: " + fileNames.size());
        System.out.println("Files that succeeded: " + success);
        System.out.println("cnt072RukA655Ruk: " + cnt072RukA655Ruk);
        System.out.println("cnt072RukA655NRuk: " + cnt072RukA655NRuk);
        System.out.println("cnt072NRukA655Ruk: " + cnt072NRukA655Ruk);
        System.out.println("cnt072RukA655RukA655NMik: " + cnt072RukA655RukA655NMik);
        System.out.println("cnt072RukA655RukA533NMik: " + cnt072RukA655RukA533NMik);
    }
}
